package com.crawler.handler;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.crawler.dao.DButils;
import com.crawler.util.HttpUtils;

public class CrewlerHandler {
	
	private static Logger logger = LoggerFactory.getLogger(CrewlerHandler.class);
	private static final String URL = "http://www.quanshuwang.com/"; 
	private static int CONNECTTIMEOUT = 3000;
	private static int READTIMEOUT = 3000;
	static int count = 32;
	
	static AtomicInteger atomicInteger = new AtomicInteger(0);
	//所有的章节
	static AtomicInteger allChapter = new AtomicInteger(0);
	public static void main(String[] args) {
		firstPage(URL);
	}

	private static void mainUrl(String url, String title) {
		try {
			long start_time = System.currentTimeMillis();
			logger.error("开始时间"+new Date(start_time));
			StringBuilder builder = getUrl(url);	
			
			Pattern p = Pattern.compile("<a.*? title=.*?href=.*?<\\/a>");
			Pattern book = Pattern.compile("target=\"_blank\" href=\"(.*?)\"");
			Matcher m = p.matcher(builder.toString());
			StringBuilder h_url = new StringBuilder();
			while (m.find()) {
				h_url.append(m.group());
			}
			Matcher m_book = book.matcher(h_url.toString());
			List<String> l_book = new ArrayList<String>();
			while (m_book.find()) {
				logger.info("第一次的地址:"+m_book.group(1));
				l_book.add(m_book.group(1));
			}	
			logger.error("一共有"+l_book.size()+"本书");
			for (String string : l_book) {
				start_url(string,title);
			}
			long end_time = System.currentTimeMillis();
			logger.error("总共下载:"+allChapter.get()+"章数据");
			logger.error("结束时间:"+new Date(end_time));
			logger.error("总共耗时:"+(end_time-start_time)/1000+"秒");
			logger.error("平均每秒下载:"+(allChapter.get()/((end_time-start_time)/1000))+"章");
		} catch (Exception e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		}
	}
	
	public static void firstPage(String url){
		StringBuilder builder;
		try {
			builder = getUrl(url);
			Pattern p = Pattern.compile("<ul class=\"channel-nav-list\">([\\s\\S]*?)</ul>");		
			Matcher m = p.matcher(builder.toString());
			StringBuilder h_url = new StringBuilder();
			while (m.find()) {
				h_url.append(m.group());
			}	
			Pattern li = Pattern.compile("<li><a href=\"(.*?)\">(.*?)</a></li>");
			Matcher m_li = li.matcher(h_url.toString());
			while (m_li.find()) {
				classifyStart(m_li.group(1),m_li.group(2));
			}				
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void classifyStart(String url,String title){
		StringBuilder builder;
		Pattern p = Pattern.compile("class=\"last\">(.*?)</a>");
		try {
			builder = getUrl(url);
			Matcher m = p.matcher(builder.toString());
			while (m.find()) {
				int size = Integer.parseInt(m.group(1));
				logger.info(title+"总共："+m.group(1)+"页");
				for (int i = 1; i <size; i++) {
					mainUrl(url.substring(0, url.indexOf("_")+1)+i+".html",title);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		} 
	}
	
	private static void start_url(String url, String title){
		Pattern p = Pattern.compile("<a.*? href=\"(.*?)\" class=\"reader\" title=.*?<\\/a>");
		try {
			Matcher m = p.matcher(getUrl(url));
			while (m.find()) {
				book_start(m.group(1),title);
			}
		}catch (Exception e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		}
	}
	
	private static void book_start(String url, String title2){
		CountDownLatch latch = null;
		ExecutorService executors = null;
		try {
			StringBuilder html = getUrl(url);
			Pattern title = Pattern.compile("<strong>(.*?)</strong>");
			Pattern author = Pattern.compile("<span class=\"r\">(.*?)</span>");
			Matcher m_title = title.matcher(html.toString());
			Matcher m_author = author.matcher(html.toString());
			String name = "";
			String authors = "";
			while (m_title.find()) {
				name = m_title.group(1);
			}
			while (m_author.find()) {
				authors = m_author.group(1);
			}
			logger.error("书名:"+name+"===开始执行....");
//			int id = DButils.insertBoookName(name, authors);
			//开线程添加内容
			Pattern tt =Pattern.compile("title=\"(.*?)\">");
			Pattern hr = Pattern.compile("href=\"(.*?)\"");
			Pattern content = Pattern.compile("<a.*?href=.*?.html.*? title=.*?<\\/a>");
			Matcher m_content = content.matcher(html.toString());
			executors = Executors.newFixedThreadPool(count); 
			List<ContentHandler> list = new ArrayList<ContentHandler>();
			while (m_content.find()) {
				String chapter = "";
				String con_url = "";
				String content_url = m_content.group();
				Matcher m_title_ = tt.matcher(content_url);
				Matcher m_url_ = hr.matcher(content_url);
				while (m_title_.find()) {
					chapter = m_title_.group(1);
				}
				while (m_url_.find()) {
					con_url = m_url_.group(1);
				}
				ContentHandler handler = new ContentHandler(title2,name, authors,chapter,url+"/"+con_url,con_url.substring(0,con_url.indexOf(".html")));
				list.add(handler);
//				getContent(id,chapter,url+"/"+con_url,con_url.substring(0,con_url.indexOf(".html")));
			}
			latch = new CountDownLatch(list.size());
			for (ContentHandler contentHandler : list) {
				contentHandler.setCount(latch);
				executors.execute(contentHandler);
			}
			latch.await();
			allChapter.addAndGet(list.size());
			logger.error("第"+atomicInteger.addAndGet(1)+"本书====书名:"+name+"==跑完了,总共有"+list.size()+"章数据");
		}catch (Exception e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		}finally{
			executors.shutdown();
		}
	}
	
	public static void getContent(int id, String chapter, String con_url,
			String sort){
	      System.out.println(Thread.currentThread().getName() + "正在执行。。。"+"章节名称:"+chapter+"====url:"+con_url);  
	        StringBuilder builder;
			try {
				builder = CrewlerHandler.getUrl(con_url);
				Pattern p = Pattern.compile("&nbsp;&nbsp;&nbsp;&nbsp;(.*?)<");
				Matcher m = p.matcher(builder.toString());
				StringBuilder connect = new StringBuilder();
				while (m.find()) {
					connect.append(m.group(1)+"\n");
				}
				DButils.insertContent(id, chapter,connect.toString(), sort);
			}catch (Exception e) {
				e.printStackTrace();
				logger.error(e.getMessage());
			}		
	}
	
	public static StringBuilder getUrl(String url2) throws MalformedURLException,
	IOException, ProtocolException, UnsupportedEncodingException,
	Exception {
		StringBuilder builder = new StringBuilder();
		int a = 0;
		try {
			URL url = new URL(url2);
			//获取连接
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
			//设置连接参数
			connection.setDoInput(true);
			connection.setDoOutput(true);
			connection.setConnectTimeout(CONNECTTIMEOUT);
			connection.setReadTimeout(READTIMEOUT);
			connection.setRequestMethod("GET");
			connection.connect();
			builder.append(new String(HttpUtils.getByte(connection.getInputStream()),"GBK"));
		} catch (Exception e) {
			if(a<=3){
				if(e.getMessage().equals("connect timed out")
						||e.getMessage().equals("Read timed out")){
					getUrl(url2);
					a++;
				}
			}else {
				logger.error(e.getMessage());
				e.printStackTrace();
			}
		}
		if(builder.toString()!=""){
			return builder;
		}else {
			throw new RuntimeException("运行时异常");
		}
		
	}

}
